import json
import os
from datetime import datetime

import logzero
import lxml.etree
import requests
from logzero import logger
from tqdm import tqdm

from config import COLLECTION_FEED
from utils import get_mongo_collection, record_run_status
import pymongo

pwd = os.path.dirname(os.path.abspath(__file__))
current_module = os.path.splitext(os.path.basename(__file__))[0]
COLLECTION_NAME = current_module
logzero.logfile(f"/tmp/{current_module}.log", maxBytes=30_000_000, backupCount=3, encoding='utf-8')

db = get_mongo_collection(COLLECTION_NAME)

insert_count = 0


def spider(feeds, need_retweeted=False):
    for feed in tqdm(feeds):
        uid = feed['uid']
        index_url = 'https://m.weibo.cn/api/container/getIndex?type=uid&value={uid}&containerid=107603{uid}&page={p}'
        p = 0
        stop = False
        while not stop:
            p += 1
            if p>20:
                logger.info('20页截止')
                break
            url = index_url.format(uid=uid, p=p)
            logger.debug(f'访问 {url}')
            resp = requests.get(url).json()
            ok = resp['ok']
            if ok == 0:
                logger.error(resp)
                logger.error('翻页结束')
                break
            cards = resp['data'].get('cards')
            for card in cards:
                if card['card_type'] == 9:
                    mid = card['mblog']['mid']
                    if filter_card(card, feed, mid, need_retweeted):
                        continue
                    card = handle_all_text(card)
                    if not card:
                        continue
                    stop = store_data(card, mid, feed)
                    if stop: break
        logger.info('完成')


def get_feeds():
    db_feed = get_mongo_collection(COLLECTION_FEED)
    cursor = db_feed.find({'purpose': '微博'}).sort("_id",pymongo.DESCENDING)
    feeds = []
    for cu in cursor:
        feeds.append({'type_': cu['type_'], 'name': cu['name'].strip(), 'cos': cu['cos'], 'uid': cu['uid'],
                      'update_time': datetime.now()})
    return feeds


def filter_card(card, feed, mid, need_retweeted):
    """
    过滤微博，如果返回True，要过滤掉
    :param card:
    :param mid:
    :return:
    """
    if card['mblog'].get('isTop') == 1:
        logger.info(f'{mid}是置顶微博，跳过')
        return True
    if feed['cos'] == 'cos':
        if not need_retweeted and card['mblog'].get('retweeted_status'):
            logger.info(f'{mid} 是转发微博， 跳过')
            return True
        is_pic = card['mblog'].get('pics')
        is_video = card['mblog'].get('page_info', {}).get('media_info', {}).get('stream_url')
        if not is_pic and not is_video:
            logger.info(f'{mid} 无图片也无视频，跳过')
            return True
        if is_pic and len(card['mblog']['pics']) >= 1:
            logger.info(f'{mid} 只有一张图片,新需求,五号也要')
            pass
        is_live = card['mblog'].get('page_info', {}).get('media_info', {}).get('goto') == 1
        if is_video and is_live:
            logger.info(f'{mid} 是直播，跳过')
            return True
    return False


def handle_all_text(card):
    # if card['mblog']['user']['id'] == 5628238455:
    #     print('5628238455')
    text_has_quanwen = '全文</a>' in card['mblog']['text']
    retweeted_has_quanwen = '全文</a>' in card['mblog'].get('retweeted_status', {}).get('text', '')
    if text_has_quanwen:
        schemeurl = card['scheme']
    if retweeted_has_quanwen:
            schemeurl = 'https://m.weibo.cn/status/' + card['mblog']['retweeted_status']['mid']
    has_quanwen = text_has_quanwen or retweeted_has_quanwen
    if has_quanwen:
        logger.info('有全文标签，获取整个text')
        text = get_all_text(schemeurl)
        if not text:
            logger.info('找不到详情，删掉记录')
            db.delete_one({'scheme': card['scheme']})
            return None
        else:
            logger.info(text)
            if text_has_quanwen:
                card['mblog']['text'] = text
            if retweeted_has_quanwen:
                card['mblog']['retweeted_status']['text'] = text
    return card


def get_all_text(url):

    resp = lxml.etree.HTML(requests.get(url).text)
    if not len(resp):
        return None
    script = resp.find('.//body/script')
    if script is None:
        return None
    js = script.text
    data = js.split('var $render_data =')[1].replace('[0] || {};', '')
    text = json.loads(data)[0]['status']['text']
    return text


def store_data(card, mid, feed):
    global insert_count
    stop = False
    old_record = db.find_one({'mblog.mid': mid})
    if old_record:
        logger.info(f'{mid} 已经存在，不再爬取此人的微博')
        stop = True
    else:
        logger.info(f'{mid} 不存在，现在插入')
        card.update(feed)
        logger.info(card)
        db.insert_one(card)
        insert_count += 1
    return stop


@record_run_status(current_module)
def main():
    feeds = get_feeds()
    spider(feeds)
    return insert_count


if __name__ == '__main__':
    main()
    # get_all_text('https://m.weibo.cn/status/4212856024792348')
